Self-Driving Car Engineer Nanodegree

In [1]:

    

# Load pickled data
import pickle

from keras.datasets import cifar10
from sklearn.model_selection import train_test_split

# TODO: Fill this in based on where you saved the training and testing data

#training_file = "traffic-signs-data/train.p"
#validation_file = "traffic-signs-data/valid.p"
#testing_file = "traffic-signs-data/test.p"

#with open(training_file, mode='rb') as f:
#    train = pickle.load(f)
#with open(validation_file, mode='rb') as f:
#    valid = pickle.load(f)
#with open(testing_file, mode='rb') as f:
#    test = pickle.load(f)
    
#X_train, y_train = train['features'], train['labels']
#X_valid, y_valid = valid['features'], valid['labels']
#X_test, y_test = test['features'], test['labels']

(X_train, y_train), (X_test, y_test) = cifar10.load_data()
X_train, X_valid, y_train, y_valid = train_test_split(X_train, y_train, test_size=0.2, random_state=0)
# y_train.shape is 2d, (50000, 1). While Keras is smart enough to handle this
# it's a good idea to flatten the array.
y_train = y_train.reshape(-1)
y_test = y_test.reshape(-1)
y_valid = y_valid.reshape(-1)


    

    




Using TensorFlow backend.

In [3]:

    

### Replace each question mark with the appropriate value. 
### Use python, pandas or numpy methods rather than hard coding the results

# TODO: Number of training examples
n_train = len(X_train)

# TODO: Number of testing examples.
n_test = len(X_test)

# TODO: What's the shape of an traffic sign image?
image_shape = X_train[0].shape

# TODO: How many unique classes/labels there are in the dataset.
n_classes = len(set(y_valid))

print("Number of training examples =", n_train)
print("Number of testing examples =", n_test)
print("Image data shape =", image_shape)
print("Number of classes =", n_classes)


    

    




Number of training examples = 40000
Number of testing examples = 10000
Image data shape = (32, 32, 3)
Number of classes = 10

In [4]:

    

### Data exploration visualization code goes here.
### Feel free to use as many code cells as needed.
import matplotlib.pyplot as plt
# Visualizations will be shown in the notebook.
%matplotlib inline

label_name = ["" for x in range(43)]
label_name[0] = "Speed limit (20km/h)"
label_name[1] = "Speed limit (30km/h)"
label_name[2] = "Speed limit (50km/h)"
label_name[3] = "Speed limit (60km/h)"
label_name[4] = "Speed limit (70km/h)"
label_name[5] = "Speed limit (80km/h)"
label_name[6] = "End of speed limit (80km/h)"
label_name[7] = "Speed limit (100km/h)"
label_name[8] = "Speed limit (120km/h)"
label_name[9] = "No passing"
label_name[10] = "No passing for vehicles over 3.5 metric tons"
label_name[11] = "Right-of-way at the next intersection"
label_name[12] = "Priority road"
label_name[13] = "Yield"
label_name[14] = "Stop"
label_name[15] = "No vehicles"
label_name[16] = "Vehicles over 3.5 metric tons prohibited"
label_name[17] = "No entry"
label_name[18] = "General caution"
label_name[19] = "Dangerous curve to the left"
label_name[20] = "Dangerous curve to the right"
label_name[21] = "Double curve"
label_name[22] = "Bumpy road"
label_name[23] = "Slippery road"
label_name[24] = "Road narrows on the right"
label_name[25] = "Road work"
label_name[26] = "Traffic signals"
label_name[27] = "Pedestrians"
label_name[28] = "Children crossing"
label_name[29] = "Bicycles crossing"
label_name[30] = "Beware of ice/snow"
label_name[31] = "Wild animals crossing"
label_name[32] = "End of all speed and passing limits"
label_name[33] = "Turn right ahead"
label_name[34] = "Turn left ahead"
label_name[35] = "Ahead only"
label_name[36] = "Go straight or right"
label_name[37] = "Go straight or left"
label_name[38] = "Keep right"
label_name[39] = "Keep left"
label_name[40] = "Roundabout mandatory"
label_name[41] = "End of no passing"
label_name[42] = "End of no passing by vehicles over 3.5 metric tons"

unique_label = set(y_valid)
maximum_traffic_signs_to_print = 20

def print_overite(title):
    print(title, end='\r')
    
def fast_draw_all_traffic_signs_in_different_images():
    size_dataset = len(y_valid)
    
    for label in unique_label:
        plt.figure(figsize=(16,0.8))
        traffic_sign_index = 0 
        number_of_signs_printed = 0
        
        for i in range(size_dataset):
            #title = "Label: " + str(label) + "/" + str(len(unique_label) - 1) + " processing " + str(i) + "/" + str(size_dataset - 1)
            #print_overite(title)
            if (y_valid[i] == label):
                traffic_signs = plt.subplot(1, maximum_traffic_signs_to_print, traffic_sign_index+1)
                traffic_signs.imshow(X_valid[i], interpolation='nearest')
                traffic_signs.axis('off')
                traffic_sign_index += 1
                
                number_of_signs_printed += 1
                if (number_of_signs_printed == maximum_traffic_signs_to_print):
                    break
        
        print(str(label+1) + "/" + str(len(unique_label)) + " - " + label_name[label])
        plt.show()

def slow_draw_all_traffic_signs_in_one_image():
    number_of_labels = len(unique_label)
    size = 0.8
    width = size * maximum_traffic_signs_to_print
    height = size * number_of_labels
    size_dataset = len(y_valid)

    print("Number of labels: " + str(number_of_labels))
    print("Total size of the image w:" + str(width) + " h:" + str(height))
    print("Total size of the dataset: " + str(size_dataset))
    print("Creating subplots, this might take a long time...")

    fig, traffic_signs = plt.subplots(number_of_labels, maximum_traffic_signs_to_print, figsize=(width,height))

    for label in range(number_of_labels):    
        traffic_sign_index = 0
        number_of_signs_printed = 0

        for i in range(size_dataset):
            title = "Processing label: " + str(label) + "/" + str(number_of_labels - 1)
            print_overite(title)

            if (y_valid[i] == label):
                traffic_signs[label][traffic_sign_index].imshow(X_valid[i], interpolation='nearest')
                traffic_signs[label][traffic_sign_index].axis('off')
                traffic_sign_index += 1

                number_of_signs_printed += 1
                if (number_of_signs_printed == maximum_traffic_signs_to_print):
                    break

    print()
    print("Painting...")

fast_draw_all_traffic_signs_in_different_images()
#slow_draw_all_traffic_signs_in_one_image()


    

    




1/10 - Speed limit (20km/h)






    













    




2/10 - Speed limit (30km/h)






    













    




3/10 - Speed limit (50km/h)






    













    




4/10 - Speed limit (60km/h)






    













    




5/10 - Speed limit (70km/h)






    













    




6/10 - Speed limit (80km/h)






    













    




7/10 - End of speed limit (80km/h)






    













    




8/10 - Speed limit (100km/h)






    













    




9/10 - Speed limit (120km/h)






    













    




10/10 - No passing






    

In [5]:

    

### Preprocess the data here. Preprocessing steps could include normalization, converting to grayscale, etc.
### Feel free to use as many code cells as needed.

#X_train, y_train = train['features'], train['labels']
#X_valid, y_valid = valid['features'], valid['labels']
#X_test, y_test = test['features'], test['labels']

import cv2
import tensorflow as tf
import numpy as np

def preprocess(x):
    x = [cv2.cvtColor(image, cv2.COLOR_RGB2GRAY) for image in x]
    return np.reshape(x, (-1, 32, 32, 1))

X_train = preprocess(X_train)
X_valid = preprocess(X_valid)
X_test = preprocess(X_test)

from sklearn.utils import shuffle

X_train, y_train = shuffle(X_train, y_train)


    

In [6]:

    

### Define your architecture here.
### Feel free to use as many code cells as needed.

from tensorflow.contrib.layers import flatten

def LeNet(x): 
    # Hyperparameters
    mu = 0
    sigma = 0.1

    # Layer 1: Convolutional. Input = 32x32x1. Output = 28x28x6.
    conv1_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 1, 6), mean = mu, stddev = sigma))
    conv1_b = tf.Variable(tf.zeros(6))
    conv1   = tf.nn.conv2d(x, conv1_W, strides=[1, 1, 1, 1], padding='VALID') + conv1_b

    # Activation.
    conv1 = tf.nn.relu(conv1)

    # Pooling. Input = 28x28x6. Output = 14x14x6.
    conv1 = tf.nn.max_pool(conv1, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

    # Layer 2: Convolutional. Output = 10x10x16.
    conv2_W = tf.Variable(tf.truncated_normal(shape=(5, 5, 6, 16), mean = mu, stddev = sigma))
    conv2_b = tf.Variable(tf.zeros(16))
    conv2   = tf.nn.conv2d(conv1, conv2_W, strides=[1, 1, 1, 1], padding='VALID') + conv2_b

    # Activation.
    conv2 = tf.nn.relu(conv2)

    # Pooling. Input = 10x10x16. Output = 5x5x16.
    conv2 = tf.nn.max_pool(conv2, ksize=[1, 2, 2, 1], strides=[1, 2, 2, 1], padding='VALID')

    # Flatten. Input = 5x5x16. Output = 400.
    fc0   = flatten(conv2)

    # Layer 3: Fully Connected. Input = 400. Output = 120.
    fc1_W = tf.Variable(tf.truncated_normal(shape=(400, 120), mean = mu, stddev = sigma))
    fc1_b = tf.Variable(tf.zeros(120))
    fc1   = tf.matmul(fc0, fc1_W) + fc1_b

    # Activation.
    fc1    = tf.nn.relu(fc1)

    # Layer 4: Fully Connected. Input = 120. Output = 84.
    fc2_W  = tf.Variable(tf.truncated_normal(shape=(120, 84), mean = mu, stddev = sigma))
    fc2_b  = tf.Variable(tf.zeros(84))
    fc2    = tf.matmul(fc1, fc2_W) + fc2_b

    # Activation.
    fc2    = tf.nn.relu(fc2)

    # Layer 5: Fully Connected. Input = 84. Output = n_classes.
    fc3_W  = tf.Variable(tf.truncated_normal(shape=(84, n_classes), mean = mu, stddev = sigma))
    fc3_b  = tf.Variable(tf.zeros(n_classes))
    logits = tf.matmul(fc2, fc3_W) + fc3_b
    
    return logits


    

In [7]:

    

### Train your model here.
### Calculate and report the accuracy on the training and validation set.
### Once a final model architecture is selected, 
### the accuracy on the test set should be calculated and reported as well.
### Feel free to use as many code cells as needed.

#Constants
EPOCHS = 10
BATCH_SIZE = 128

# Features and Labels
x = tf.placeholder(tf.float32, (None, 32, 32, 1))
y = tf.placeholder(tf.int32, (None))
one_hot_y = tf.one_hot(y, n_classes)

#Training Pipeline
rate = 0.001

logits = LeNet(x)
cross_entropy = tf.nn.softmax_cross_entropy_with_logits(logits, one_hot_y)
loss_operation = tf.reduce_mean(cross_entropy)
optimizer = tf.train.AdamOptimizer(learning_rate = rate)
training_operation = optimizer.minimize(loss_operation)

#Model Evaluation
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))
accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
saver = tf.train.Saver()

def evaluate(X_data, y_data):
    num_examples = len(X_data)
    total_accuracy = 0
    sess = tf.get_default_session()
    for offset in range(0, num_examples, BATCH_SIZE):
        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]
        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y})
        total_accuracy += (accuracy * len(batch_x))
    return total_accuracy / num_examples

#Train the Model
with tf.Session() as sess:
    sess.run(tf.global_variables_initializer())
    num_examples = len(X_train)
    
    print("Training...")
    print()
    for i in range(EPOCHS):
        X_train, y_train = shuffle(X_train, y_train)
        for offset in range(0, num_examples, BATCH_SIZE):
            end = offset + BATCH_SIZE
            batch_x, batch_y = X_train[offset:end], y_train[offset:end]
            sess.run(training_operation, feed_dict={x: batch_x, y: batch_y})
                
        print("EPOCH {}".format(i+1) + ", accuracy: {:.3f}".format(evaluate(X_valid, y_valid)))
        
    saver.save(sess, './lenet')
    print("Model saved")


    

    




Training...

EPOCH 1, accuracy: 0.295
EPOCH 2, accuracy: 0.383
EPOCH 3, accuracy: 0.423
EPOCH 4, accuracy: 0.475
EPOCH 5, accuracy: 0.503
EPOCH 6, accuracy: 0.514
EPOCH 7, accuracy: 0.541
EPOCH 8, accuracy: 0.548
EPOCH 9, accuracy: 0.533
EPOCH 10, accuracy: 0.572
Model saved

In [8]:

    

#Evaluate accuracy of the system

def evaluate(X_data, y_data):
    num_examples = len(X_data)
    total_accuracy = 0
    sess = tf.get_default_session()
    for offset in range(0, num_examples, BATCH_SIZE):
        batch_x, batch_y = X_data[offset:offset+BATCH_SIZE], y_data[offset:offset+BATCH_SIZE]
        accuracy = sess.run(accuracy_operation, feed_dict={x: batch_x, y: batch_y})
        total_accuracy += (accuracy * len(batch_x))
    return total_accuracy / num_examples

#Model Evaluation
correct_prediction = tf.equal(tf.argmax(logits, 1), tf.argmax(one_hot_y, 1))
accuracy_operation = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))

def evaluate_accuracy(kind, x, y):
    with tf.Session() as sess:
        saver.restore(sess, tf.train.latest_checkpoint('.'))
    
        accuracy = evaluate(x, y)
        print(kind + " Accuracy = {:.3f}".format(accuracy))
        
evaluate_accuracy("Validation", X_valid, y_valid)
evaluate_accuracy("Training", X_train, y_train)
evaluate_accuracy("Test", X_test, y_test)


    

    




Validation Accuracy = 0.572
Training Accuracy = 0.646
Test Accuracy = 0.573

In [ ]:

    

### Load the images and plot them here.
### Feel free to use as many code cells as needed.

import matplotlib.image as mpimg
    
#Images from http://benchmark.ini.rub.de/?section=gtsrb&subsection=dataset#Downloads
#newImageFileNames = [
#    "new-images/00000.ppm",
#    "new-images/00001.ppm",
#    "new-images/00002.ppm",
#    "new-images/00003.ppm",
#    "new-images/00004.ppm"
#]
#y_newImages = [16, 1, 38, 33, 11]

#Images from Google search https://www.google.co.uk/search?q=german+road+signs
newImageFileNames = [
    "random-web-images/label_1.jpg",
    "random-web-images/label_17.jpg",
    "random-web-images/label_18.jpg",
    "random-web-images/label_25.jpg",
    "random-web-images/label_28.jpg"
]
y_newImages = [1, 17, 18, 25, 28]

newImages = [mpimg.imread(newImageFileName) for newImageFileName in newImageFileNames]

def displayImages(images):
    size = len(images)
    for i in range(size):
        image = plt.subplot(1, size, i+1)
        image.imshow(images[i])    

displayImages(newImages)


    

In [ ]:

    

### Run the predictions here and use the model to output the prediction for each image.
### Make sure to pre-process the images with the same pre-processing pipeline used earlier.
### Feel free to use as many code cells as needed.

#Pre-Process images
newImages = [cv2.resize(newImage, (32, 32)) for newImage in newImages]

X_newImages = preprocess(newImages).astype(np.float32)

def predict_signs(X):
    with tf.Session() as sess:
        saver.restore(sess, tf.train.latest_checkpoint('.'))
        softmax = tf.nn.softmax(logits)
        label_predictions = sess.run(softmax, feed_dict={x: X})    
        return [np.argmax(label_prediction) for label_prediction in label_predictions]

predicted_labels = predict_signs(X_newImages)

print("Predicted labels: " + str(predicted_labels))
print("Correct labels: " + str(y_newImages))


    

In [ ]:

    

### Calculate the accuracy for these 5 new images. 
### For example, if the model predicted 1 out of 5 signs correctly, it's 20% accurate on these new images.

def ratio_correct_matches(predicted_labels, correct_labels):
    number_of_correct_matches = 0
    number_of_labels = len(predicted_labels)

    for i in range(number_of_labels):
        if (predicted_labels[i] == correct_labels[i]):
            number_of_correct_matches += 1

    return number_of_correct_matches * 100 / number_of_labels

ratio_success = ratio_correct_matches(predicted_labels, y_newImages)

print("Accuracy is " + str(ratio_success) + "%")


    

In [ ]:

    

### Print out the top five softmax probabilities for the predictions on the German traffic sign images found on the web. 
### Feel free to use as many code cells as needed.

def softmax_top_probabilities(X, number_of_probabilities):
    with tf.Session() as sess:
        saver.restore(sess, tf.train.latest_checkpoint('.'))
        softmax = tf.nn.softmax(logits)
        softmax_probabilities = sess.run(softmax, feed_dict={x: X})    
        top_probabilities = tf.nn.top_k(softmax_probabilities, number_of_probabilities)
        return sess.run(top_probabilities)

top_probabilities = softmax_top_probabilities(X_newImages, 5)

for i in range(len(X_newImages)):
    print("Top five softmax probabilities for image number " + str(i+1))
    print(top_probabilities.values[i])
    print("-> top five labels: " + str(top_probabilities.indices[i]))
    print()


    

In [ ]:

    

#Traffic sign General Caution (label 18) analysis

problematic_label = 18

general_caution = []
general_caution_label = []
for i in range(len(y_test)):
    if (y_test[i] == problematic_label):
        general_caution.append(X_test[i])
        general_caution_label.append(problematic_label)
        
evaluate_accuracy("General Caution", general_caution, general_caution_label)